{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Searching and Ranking"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 如何检索网页\n",
    "- 建立索引\n",
    "- 对网页进行搜索\n",
    "- 结果排名"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sqlite3 import dbapi2 as sqlite_version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class crawler:\n",
    "    def __init__(self, dbname):\n",
    "        pass\n",
    "    \n",
    "    def __del__(self):\n",
    "        pass\n",
    "    \n",
    "    def dbcommit(self):\n",
    "        pass\n",
    "    \n",
    "    def getentryid(self, table, filed, value, createnew=True):\n",
    "        return None\n",
    "    \n",
    "    def addtoindex(self, url, soup):\n",
    "        print('indexing %s' %url)\n",
    "        \n",
    "    def gettextonly(self, soup):\n",
    "        return None\n",
    "    \n",
    "    def separatewords(self, text):\n",
    "        return None\n",
    "    \n",
    "    def isindexed(self, url):\n",
    "        return False\n",
    "    \n",
    "    def addlinkref(self, urlFrom, urlTo, linkText):\n",
    "        pass\n",
    "    \n",
    "    def crawl(self, pages, depth=2):\n",
    "        pass\n",
    "    \n",
    "    def creatindextables(self):\n",
    "        pass\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A Simple crawler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Python\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\urllib3\\connectionpool.py:852: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'<!DOCTYPE html>\\n<html lang=\"en\">\\n<head>\\n\\t<meta cha'\n"
     ]
    }
   ],
   "source": [
    "import urllib3\n",
    "c_http = urllib3.PoolManager()\n",
    "c = c_http.request('GET','https://kiwitobes.com/')\n",
    "contents = c.data\n",
    "print(contents[0:50])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
